package chapter03
import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
object Test30_Ads {
  def main(args: Array[String]): Unit = {
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    val ads = new SparkConf().setMaster("local[*]").setAppName("ads")
    val sc = new SparkContext(ads)
    //读取文件
    val value = sc.textFile("input/agent.log")
    //切分
    println(value.map(e=>e.split(" "))
      .map(e=>(e(1),e(4)))
      .map(e=>(e,1))
      .reduceByKey(_+_)
      .map(e=>(e._1._1,(e._1._2,e._2)))
      .groupByKey()
      .map(e=>(e._1,e._2.toList.sortBy(f=>f._2).reverse))
      .map(e=>(e._1,e._2.take(3)))
      .collect()
      .toList
    )
    sc.stop()
  }
}
